{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Rental Listing Inquiries\n",
    "## 刘晓明 第三周作业（part 2-2）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第三步：调整正则化参数：reg_alpha 和reg_lambda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先 import 必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第1-2步参数调整得到的n_estimators最优值（220）， 'max_depth': 5, 'min_child_weight':1，其余参数继续默认值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用交叉验证评价模型性能时，用scoring参数定义评价指标。评价指标是越高越好，因此用一些损失函数当评价指标时，需要再加负号，如neg_log_loss，neg_mean_squared_error 详见sklearn文档：http://scikit-learn.org/stable/modules/model_evaluation.html#log-loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.1, 1, 1.5, 2], 'reg_lambda': [0.1, 0.5, 1, 2]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [0.1,1,1.5,2]        #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.1,0.5,1,2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58521, std: 0.00355, params: {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58512, std: 0.00306, params: {'reg_alpha': 0.1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58545, std: 0.00348, params: {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  mean: -0.58584, std: 0.00340, params: {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  mean: -0.58477, std: 0.00345, params: {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58480, std: 0.00330, params: {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58447, std: 0.00322, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.58500, std: 0.00319, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.58490, std: 0.00308, params: {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       "  mean: -0.58508, std: 0.00283, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58535, std: 0.00302, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58487, std: 0.00327, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58497, std: 0.00290, params: {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  mean: -0.58513, std: 0.00302, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58528, std: 0.00327, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58539, std: 0.00337, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1, 'reg_lambda': 1},\n",
       " -0.5844748117599161)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.7,\n",
    "        colsample_bytree=0.6,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  189.58464355,   196.34483027,   197.19507895,   198.17073469,\n",
       "          198.33694415,   198.4499507 ,   198.43074946, 17471.34742632,\n",
       "          186.17864871,   194.28951263,   201.19370756,   202.86260304,\n",
       "          204.16327744,   200.51686912,   200.70087943,   191.16133389]),\n",
       " 'mean_score_time': array([0.57043266, 0.58563352, 0.6068347 , 0.57943311, 0.57603297,\n",
       "        0.58403335, 0.61363516, 0.55683188, 0.57203283, 0.58403349,\n",
       "        0.58663363, 0.6188354 , 0.60363455, 0.59463387, 0.59883423,\n",
       "        0.51362939]),\n",
       " 'mean_test_score': array([-0.58520982, -0.58511776, -0.58544582, -0.58584143, -0.58476674,\n",
       "        -0.58479605, -0.58447481, -0.58500198, -0.5849036 , -0.58507582,\n",
       "        -0.58534841, -0.58487383, -0.58496926, -0.58512582, -0.58528279,\n",
       "        -0.58539047]),\n",
       " 'mean_train_score': array([-0.50789754, -0.50994553, -0.51193761, -0.5149528 , -0.51088227,\n",
       "        -0.51232879, -0.51405386, -0.51639762, -0.51320164, -0.51481042,\n",
       "        -0.51599632, -0.51776721, -0.51585838, -0.51687944, -0.5181341 ,\n",
       "        -0.51976496]),\n",
       " 'param_reg_alpha': masked_array(data=[0.1, 0.1, 0.1, 0.1, 1, 1, 1, 1, 1.5, 1.5, 1.5, 1.5, 2,\n",
       "                    2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.1, 0.5, 1, 2, 0.1, 0.5, 1, 2, 0.1, 0.5, 1, 2, 0.1,\n",
       "                    0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([11,  9, 15, 16,  2,  3,  1,  7,  5,  8, 13,  4,  6, 10, 12, 14]),\n",
       " 'split0_test_score': array([-0.57858714, -0.57944958, -0.5790797 , -0.57983376, -0.57855445,\n",
       "        -0.5788406 , -0.57870727, -0.57917014, -0.57926088, -0.58040958,\n",
       "        -0.57988915, -0.57889063, -0.57980584, -0.57949685, -0.57921769,\n",
       "        -0.57906734]),\n",
       " 'split0_train_score': array([-0.50849171, -0.51015414, -0.51260989, -0.51577744, -0.51121683,\n",
       "        -0.51278674, -0.51506304, -0.51732498, -0.51413894, -0.51599219,\n",
       "        -0.51669661, -0.51936168, -0.5170326 , -0.51794394, -0.51925101,\n",
       "        -0.52093049]),\n",
       " 'split1_test_score': array([-0.58487917, -0.58462146, -0.58489637, -0.58478966, -0.58421264,\n",
       "        -0.58430046, -0.58401257, -0.58431275, -0.5846265 , -0.5844116 ,\n",
       "        -0.58492674, -0.58400745, -0.58409454, -0.58482788, -0.58481987,\n",
       "        -0.58543116]),\n",
       " 'split1_train_score': array([-0.50817922, -0.51072765, -0.51179613, -0.51524366, -0.51169367,\n",
       "        -0.51239211, -0.51504516, -0.51667827, -0.51382636, -0.5153166 ,\n",
       "        -0.51641834, -0.51823016, -0.51615259, -0.51705225, -0.51869221,\n",
       "        -0.52015244]),\n",
       " 'split2_test_score': array([-0.58618342, -0.58624973, -0.58620461, -0.58659327, -0.58532887,\n",
       "        -0.58536328, -0.58501271, -0.58596875, -0.58539736, -0.58450989,\n",
       "        -0.58566927, -0.58637283, -0.58572155, -0.58590762, -0.58643431,\n",
       "        -0.58593598]),\n",
       " 'split2_train_score': array([-0.50769195, -0.50955585, -0.51159975, -0.51452377, -0.51066441,\n",
       "        -0.5125055 , -0.51398294, -0.5162748 , -0.51333715, -0.51426189,\n",
       "        -0.51604005, -0.51756953, -0.51585858, -0.51752319, -0.51801851,\n",
       "        -0.51992561]),\n",
       " 'split3_test_score': array([-0.58783369, -0.5880986 , -0.58885846, -0.58864386, -0.58860954,\n",
       "        -0.58843579, -0.5883278 , -0.58775534, -0.5880661 , -0.58799732,\n",
       "        -0.58817511, -0.58796271, -0.58781757, -0.58779583, -0.58861311,\n",
       "        -0.5885075 ]),\n",
       " 'split3_train_score': array([-0.50841208, -0.51028443, -0.51244189, -0.51489479, -0.51122801,\n",
       "        -0.51235014, -0.51336328, -0.51634071, -0.51343978, -0.5149362 ,\n",
       "        -0.51579687, -0.51806336, -0.51563536, -0.51675952, -0.51821524,\n",
       "        -0.51988874]),\n",
       " 'split4_test_score': array([-0.58856668, -0.58717005, -0.5881908 , -0.58934767, -0.58712893,\n",
       "        -0.58704079, -0.58631426, -0.58780379, -0.58716782, -0.58805162,\n",
       "        -0.5880826 , -0.58713621, -0.58740754, -0.58760166, -0.58732957,\n",
       "        -0.58801116]),\n",
       " 'split4_train_score': array([-0.50671274, -0.50900556, -0.51124038, -0.51432432, -0.50960843,\n",
       "        -0.51160944, -0.51281491, -0.51536935, -0.51126595, -0.51354521,\n",
       "        -0.51502972, -0.51561131, -0.51461278, -0.51511829, -0.51649351,\n",
       "        -0.5179275 ]),\n",
       " 'std_fit_time': array([3.44670780e+00, 1.59154980e+00, 1.02698145e+00, 1.72535604e+00,\n",
       "        1.39838468e+00, 1.83332944e+00, 2.67467683e+00, 8.63423911e+03,\n",
       "        3.34897861e+00, 5.78677417e+00, 6.26097013e+00, 4.50806682e+00,\n",
       "        7.26598344e+00, 1.92665874e+00, 2.61558501e+00, 9.99882287e+00]),\n",
       " 'std_score_time': array([0.02673424, 0.01578192, 0.06582294, 0.01201088, 0.00695747,\n",
       "        0.01091845, 0.05456159, 0.01205723, 0.01835312, 0.01613794,\n",
       "        0.0160582 , 0.03349464, 0.01409471, 0.01290965, 0.01646174,\n",
       "        0.06687279]),\n",
       " 'std_test_score': array([0.00355164, 0.00305793, 0.00348102, 0.00340325, 0.00345151,\n",
       "        0.00329615, 0.00322486, 0.0031891 , 0.0030754 , 0.00282579,\n",
       "        0.0030183 , 0.00326979, 0.00290003, 0.00302142, 0.00327439,\n",
       "        0.00337243]),\n",
       " 'std_train_score': array([0.0006547 , 0.00060091, 0.00051508, 0.00051905, 0.00071554,\n",
       "        0.00039061, 0.00089644, 0.00063473, 0.00100904, 0.00084501,\n",
       "        0.0005735 , 0.0012273 , 0.00078315, 0.00096886, 0.00092417,\n",
       "        0.00099257])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.584475 using {'reg_alpha': 1, 'reg_lambda': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\LXM\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsnXd8VFX6/99POiQklICEhE6QSAi9hY5KkQ7qCqyKq2BZ1lW/a1tXl0V3bVvUVRcsixVlf4KU0AQFKVKlBEKAIC0JoSWQENKT5/fHnUAIIaTMMJPkvF+vvJg795w7507IfOY8VVQVg8FgMBjsjZuzF2AwGAyG6okRGIPBYDA4BCMwBoPBYHAIRmAMBoPB4BCMwBgMBoPBIRiBMRgMBoNDMAJjMBgMBodgBMZgMBgMDsEIjMFgMBgcgoezF+BMAgMDtUWLFs5ehsFgMFQpfv7557Oq2vB642q0wLRo0YLt27c7exkGg8FQpRCRY2UZZ0xkBoPBYHAIRmAMBoPB4BCMwBgMBoPBIdRoH0xJ5ObmkpCQQFZWlrOXYigBHx8fQkJC8PT0dPZSDAbDdTACU4yEhATq1KlDixYtEBFnL8dQBFUlOTmZhIQEWrZs6ezlGAyG62BMZMXIysqiQYMGRlxcEBGhQYMGZndpMFQRjMCUgBEX18X8bgyGqoMRGIPBYKhBnLuYwxebj3Es+aLDX8v4YAwGg6Gak5Wbz+rYUyzceYIfD54mN1/504gwHurXyqGva3YwNZy1a9cycuTISo8pDVXl8ccfp02bNkRERLBjx44Sx73wwgs0bdoUPz+/Cr+WwWCwyC9QNsSd5Q//bzfdXlnN9Lk7iU44z5TIFkT9ri8P9nV8oIzZwbg4qoqq4uZWdb8LLF++nLi4OOLi4tiyZQuPPvooW7ZsuWrcqFGjmD59OqGhoU5YpcFQ9VFVYk6ksXBnIot3n+D0hWz8vD0YHt6YsZ2D6dWqAe5uN86P6VCBEZFhwNuAO/CRqr5W7PwU4E0g0fbUu6r6ke3cG8AIrF3WKuD3qqoishYIAjJtc4ao6mkRaQZ8CtS1vd5zqrqsMuv/y5IY9p1Iq8wlruKWJv78eVT7UsccPXqU4cOHM2jQIDZt2sQTTzzBrFmzyM7OpnXr1syZMwc/Pz+WLVvGU089RWBgIF26dOHw4cNERUWVeM2tW7fyxBNPkJmZSa1atZgzZw4333zzFWNmzJjBL7/8QmJiIvHx8TzzzDNMnToVgPT0dO6880727t1L165d+eKLLxARZs6cyZIlS8jMzCQyMpLZs2df5YhftGgR9913HyJCr169OH/+PElJSQQFBV0xrlevXuV9Ow0GAxCfksHi3Sf4dmcih06n4+kuDGjbiHGdg7k1rBE+nu5OWZfDBEZE3IH3gNuBBGCbiCxW1X3Fhs5T1enF5kYCfYAI21MbgAHAWtvxZFUtXqXyT8D/VPU/InILsAxoYafbueEcOHCAOXPmMHPmTMaPH8/q1avx9fXl9ddf55///CfPPPMMDz/8MOvWraNly5ZMnDix1Ou1a9eOdevW4eHhwerVq/njH//I/PnzrxoXHR3N5s2buXjxIp07d2bEiBEA7Ny5k5iYGJo0aUKfPn3YuHEjffv2Zfr06bz00ksA3HvvvURFRTFq1ChmzZoFwCOPPEJiYiJNmza99BohISEkJiZeJTAGg6HsnLuYw9I9SSzalci2o+cA6N6iHq+MDWdEhyDq+Xo5eYWO3cH0AA6p6mEAEfkaGAMUF5iSUMAH8AIE8AROlWGOv+1xAHCiAmu+guvtNBxJ8+bN6dWrF1FRUezbt48+ffoAkJOTQ+/evdm/fz+tWrW6lHA4ceJEPvjgg2teLzU1lfvvv5+4uDhEhNzc3BLHjRkzhlq1alGrVi0GDRrE1q1bqVu3Lj169CAkJASATp06cfToUfr27cuaNWt44403yMjIICUlhfbt2zNq1CgeeeSRS9dU1atex4QbGwzlJys3n+9jT/PtzsRLzvo2jfx4eujNjO7YhKb1azt7iVfgSIEJBuKLHCcAPUsYN0FE+gMHgSdVNV5VN4nIGiAJS2DeVdXYInPmiEg+MB94Ra1PsBnAdyLyO8AXuK2kRYnINGAaQLNmzSpzfw7F19cXsD6cb7/9dr766qsrzu/cubNc13vxxRcZNGgQ3377LUePHmXgwIEljiv+wV947O3tfek5d3d38vLyyMrK4rHHHmP79u00bdqUGTNmlJgEGRISQnz85f8KCQkJNGnSpFzrNxhqKvkFyubDyXy7M5EVe0+Snp1HozreTIlswZhOwbRv4u+yX9gc6Tku6Y6Lf5VdArRQ1QhgNZYPBRFpA4QBIVhCNdgmQmCZxzoA/Ww/99qenwh8oqohwB3A5yJy1f2p6geq2k1VuzVseN1+OU6nV69ebNy4kUOHDgGQkZHBwYMHadeuHYcPH+bo0aMAzJs3r9TrpKamEhwcDMAnn3xyzXGLFi0iKyuL5ORk1q5dS/fu3a85tlBMAgMDSU9P55tvvilx3OjRo/nss89QVTZv3kxAQIAxjxkMpaCq7E1M5a9L9xH52vdM/mgLK/aeZFh4Y758qCebnr+VF0bcQnhwgMuKCzh2B5MANC1yHEIxs5WqJhc5/BB43fZ4HLBZVdMBRGQ50AtYp6qJtrkXRGQulinuM+BBYJjt3CYR8QECgdN2vq8bSsOGDfnkk0+YOHEi2dnZALzyyiu0bduW999/n2HDhhEYGEiPHj1Kvc4zzzzD/fffzz//+U8GDx58zXE9evRgxIgRHD9+nBdffJEmTZpw8ODBEsfWrVuXqVOn0qFDB1q0aHGFGBX1wdxxxx0sW7aMNm3aULt2bebMmXNpXKdOndi1a9elNc6dO5eMjAxCQkJ46KGHmDFjRpneJ4OhOlDorF+4M5G40+l4uAkDb27EiyObcFvYTU5z1lcUKck+bpcLi3hgmb1uxYoS2wZMUtWYImOCVDXJ9ngc8Kyq9hKRXwFTsQRDgBXAW8ByoK6qnhURT+ArYLWqzrKJ0DxV/UREwoDvgWAt5Qa7deumxTtaxsbGEhYWZqd3wbGkp6fj5+eHqvLb3/6W0NBQnnzyyQpfb8aMGfj5+fGHP/zBjqu0P1Xpd2QwXI9rOevHdAp2GWd9cUTkZ1Xtdr1xDtvBqGqeiEwHVmKFDf9XVWNEZCawXVUXA4+LyGggD0gBptimfwMMBvZgmdVWqOoSEfEFVtrExR3LrPahbc7/AR+KyJO2OVNKE5fqwIcffsinn35KTk4OnTt35uGHH3b2kgwGQxmoas76iuKwHUxVoKrvYEpizpw5vP3221c816dPH9577z0nrcj+VPXfkaFmUuisX2hz1l+wOetHd2zC2M6u7awvjtN3MAbn8MADD/DAAw84exkGg4HLmfWLdlmZ9afSrMz6YeGNGdspmN6tb2xm/Y3GCIzBYDDYmermrK8oRmAMBoPBDpzPsJz1C3dedtZ3a+5amfU3GiMwBoPBUEEKnfULdyWy9sBlZ/0fhrRlTKfgauOsryhGYAwGg6EcXMtZf3/vFlXOWe9oqm4NeINduBH9YPbv30/v3r3x9vbm73//e4WvYzA4C8tZf2Vm/fK9Jxka3pgvHrQy6/800vUz6280Zgfj4lSHfjD169fnnXfeYeHChc5eisFQLhLOZbBol3HWVxQjMKWx/Dk4uce+12zcAYa/VuqQ6tYPplGjRjRq1IilS5dW4o0zGG4Mhc76RTtPsPVoCmCc9RXFCIyLUp36wRgMro5x1jsGIzClcZ2dhiOpTv1gDAZXJL9A2VKkDL5x1tsfIzAuSnXqB2MwuAqqyr6kNBbtOsGiXYk1LrP+RlN1Pcc1hOrQD8ZgcDYJ5zJ4b80hhvxrHSPe2cB/NxyhQ3AA707qzPY/3cbf7+pI39BAIy52xuxgXJzq0A/m5MmTdOvWjbS0NNzc3HjrrbfYt28f/v7+JV7XYLAHxlnvfEw15SpcTdn0gzEYriQrN58f9ltl8Aud9a0b+jKuc7Bx1tsRU025BmD6wRgMl531C3clsnyPcda7EmYHU4V3MCVh+sEYagJFnfWLd53gZFoWft4eDG3fmHGdjbPe0ZgdTA3F9IMxVGcKM+sX7Urk4KnCzPqGvDAijNvCbqKWl8msdyWMwBgMBpfmfEYOy/acZOHOxCuc9S/bnPX1jbPeZTECYzAYXI5CZ/3CnYmsKeKsN5n1VQsjMAaDwSUoyVnfsI439/VuwTjjrK+SODTRUkSGicgBETkkIs+VcH6KiJwRkV22n4eKnHtDRGJEJFZE3hHb/ywRWWu7ZuGcRkXm3C0i+2zz5jry3qoLrlSuf8qUKbRs2ZJOnTrRqVMndu3aVeHXNFQNVJV9J9L427JY+rz2A5M+2sKyPScZ0t4qg7/5+Vt50ZTBr7I4bAcjIu7Ae8DtQAKwTUQWq+q+YkPnqer0YnMjgT5AhO2pDcAAYK3teLKqbi82JxR4HuijqueKCk9VpqaV63/zzTe58847b8CqDM4k8Xwmi3YlsnCncdZXZxz5qdUDOKSqh1U1B/gaGFPGuQr4AF6AN+AJnLrOnKnAe6p6DkBVT1do1S7A0aNHCQsL47HHHqNLly58/vnn9O7dmy5dunDXXXeRnp4OwLJly2jXrh19+/bl8ccfL3WXsXXrViIjI+ncuTORkZEcOHDgqjEzZszg3nvvZfDgwYSGhvLhhx9eOldYrr9du3ZMnjyZwvD2mTNn0r17d8LDw5k2bRolhb03atSI7t274+npWdm3xlCFSc3IZe6W49w9exN9XvuBN1YcoI6PJy+PDWfrC7fx0f3dGdWxiRGXaoQjfTDBQHyR4wSgZwnjJohIf+Ag8KSqxqvqJhFZAyQBAryrqrFF5swRkXxgPvCKWp9qbQFEZCPgDsxQ1RXFX0xEpgHTAJo1a1bqDby+9XX2p+wv082WlXb12/Fsj2evO66mlut/4YUXmDlzJrfeeiuvvfbaFUU2DVWPos76tQfOkJNfYJz1NQhHCkxJBtPiX2+XAF+paraIPAJ8CgwWkTZAGBBiG7dKRPqr6jos81iiiNTBEph7gc+w7iUUGGibt15EwlX1/BULUP0A+ACsREs73KdDqInl+l999VUaN25MTk4O06ZN4/XXX78kXoaqQ0GBsvmI1bN++d6TXMiynPX39m5unPU1DEcKTALQtMhxCHCi6ABVTS5y+CHwuu3xOGCzqqYDiMhyoBewTlUTbXMv2Bz5PbAEJsE2Jxc4IiIHsARnW0VvoCw7DUdRE8v1BwUFXXqtBx54oNSAAINroarEJl1g0a5EFtky63293BkWHmQy62swjvTBbANCRaSliHgB9wCLiw4QkaAih6OBQjPYcWCAiHiIiCeWgz/Wdhxom+sJjAT22uYsBAbZzgVimcwOO+TObiA1qVx/UlISYH1YLVy4kPDw8Epdz+B4Es9n8v7aQwx7az13vLOejzccITzYn39P7Mz2P93OP+42ZfBrMg7bwahqnohMB1Zi+UT+q6oxIjIT2K6qi4HHRWQ0kAekAFNs078BBgN7sMxqK1R1iYj4Aitt4uIOrMba+WB7nSEisg/IB54utkOqklT3cv133HEHH330EU2aNGHy5MmcOXMGVaVTp06XrmFwLVIzclm6J4mFuxLZesTKrO9qMusNJWCKXVbhYpemXL/hRpGVm8+aS2XwLzvrTRn8mokpdlkDMOX6DY6k0Fm/aOcJlu1NMs56Q7kxO5gqvIMpCVOu31BZYpPSWLjzamf92M5NiGxt/CkGs4OpsZhy/YaKUJhZv2jnCQ6cuoCHmzCgrcmsN1QOIzAGQw0lNSOXZXuT+HancdYbHIMRGIOhBlHorF+4K5E1+y876//vdiuzvlkD46w32A8jMAZDNac0Z/3YTsGEBxtnvcExGIExGKophc76xbtPkJRqnPWGG0/VrQFvsAs3oh/Ml19+SUREBBEREURGRrJ79+4KX8tQOonnM/nP2l8Y+q91DH/byqy/JejKzPp+oQ2NuBhuCGYH4+JUh34wLVu25Mcff6RevXosX76cadOmsWXLFmcvq9pwTWf9mPaMiGhinPUGp2EExgU5evQow4cPZ9CgQWzatIknnniCWbNmkZ2dTevWrZkzZw5+fn4sW7aMp556isDAQLp06cLhw4eJiooq8Zpbt27liSeeIDMzk1q1ajFnzhxuvvnmK8bMmDGDX375hcTEROLj43nmmWeYOnUqcLkfzN69e+natStffPEFIsLMmTNZsmQJmZmZREZGMnv27Kvs+ZGRkZce9+rVi4SEBDu/YzWPkpz1rYyz3uBiGIEphZN/+xvZsfbtB+Md1o7Gf/zjdcdV134wH3/8McOHDy/Te2W4koICZcuRFBbuTDTOekOVwAiMi1Id+8GsWbOGjz/+mA0bNlT27alRGGe9oapiBKYUyrLTcBTVrR9MdHQ0Dz30EMuXL6dBgwblWntN5MT5TBbtOsHCnYlXZNb/8Q6TWW+oOlRdz3ENoTr0gzl+/Djjx4/n888/p23btqWusyaTmpHLV1uP86vZm4h87QdeX7EfPx8PXh7Tnq0v3MbHU0zPekPVwuxgXJzq0A9m5syZJCcn89hjjwHg4eFB8SKjNZWs3HzWHrDK4BtnvaG6YaopV+FqyqYfTNVE9bKzfumey8760R2bGGe9oUpgqinXAEw/mKrHgZMX+PPivWw+nIKvlztDwxtbPetbNcDD3VisDdULs4OpwjuYkjD9YFyTtKxc3loVx6ebjuLn7cH/DWnLXV2bGn+KoUpidjA1FNMPxrUoKFDm70jg9RX7Sb6Yw8QezfjDkJtNdr2hRuDQPbmIDBORAyJySESeK+H8FBE5IyK7bD8PFTn3hojEiEisiLwjNqO0iKy1XbNwTqNi17xTRFRErquu16Im7+pcnar0u9mTkMqEWT/x9DfRNK1fm8W/7cvfxnUw4mKoMThsByMi7sB7wO1AArBNRBar6r5iQ+ep6vRicyOBPkCE7akNwABgre14sqpeFYYkInWAx4EKF7ry8fEhOTmZBg0aGEeri6GqJCcn4+Pj4+yllErKxRzeXHmAr7cdp4GvF3+/qyPjOwfjZhIiDTUMR5rIegCHVPUwgIh8DYwBigtMSSjgA3gBAngCp8ow72XgDaDCYVAhISEkJCRw5syZil7C4EB8fHwuVRRwNfILlLlbjvH37w6Snp3Hb/q05Pe3heLv4+nspRkMTsGRAhMMxBc5TgB6ljBugoj0Bw4CT6pqvKpuEpE1QBKWwLyrqrFF5swRkXxgPvCKqqqIdAaaqmqUiFRYYDw9PS+VXzEYysr2oym8tCiGfUlp9G7VgL+MaU/bm+o4e1kGg1NxpMCUZA8obkBfAnylqtki8gjwKTBYRNoAYUDhV9VVItJfVddhmccSbeaw+cC9IvIF8C9gynUXJTINmAbQrFmzCtyWwXCZ02lZvLZ8Pwt2JhIU4MO7kzozokOQMa8aXBJVJf/8efJOn8EjsAEeDi7b5EiBSQCaFjkOAU4UHaCqyUUOPwRetz0eB2xW1XQAEVkO9ALWqWqibe4FEZmLZYpbBIQDa21/2I2BxSIyurivRlU/AD4AK0zZDvdpqIHk5hfwycajvP19HDl5BUwf1IbHBrWmtpcJzDTceDQ3l7yUFPJOnyHvTCk/yclgK3TbeMafqXfPPQ5dlyP/GrYBoSLSEkgE7gEmFR0gIkGqmmQ7HA0UmsGOA1NF5FWsndAA4C0R8QDqqupZEfEERgKrVTUVCCxy3bXAH0oKBDAYKsv6uDPMWBzDL2cuMrhdI14aeQstAn2dvSxDNaQgM9MShrNnSxWP/HPnoIQIS/e6dfFo2BCPhg3xbtUKj0YNLx37dIgo4RXti8MERlXzRGQ6sBJwB/6rqjEiMhPYrqqLgcdFZDSQB6Rw2cT1DTAY2INlVluhqktExBdYaRMXd2A11s7HYHA4Cecy+OvSWJbvPUnzBrX5+P5u3Bp2k7OXZahiqCoFaWmXBaIU8ShIT7/6Au7ueAQG4hEYiGdQELUiIizRKCIeHg0b4tGgAeLl3JB4k8lvii4arkNWbj4frDvM+2utitbTB7XhoX6t8PE0WfiGy2h+PvkpKVeKREnicfYsaitcWxTx8bksDoGBV4pFEfFwr1cPcXILdZPJbzBUElVldexpXo7ax/GUDEZ0COKPI8IIrlvL2Usz3EAKcnLIt4lDbqFJ6uxZSyyKikdKCuTnXzXfzd//kjjU6tLlmuLh5udX7YJDjMAYDCVw5OxF/rIkhrUHztCmkR9fPtSTPm0Crz+xjOTm5+LpbvJjnEl++kXyzpy+2p9RKB42AclPTb16sgjuDRrYRCIQ77B2tp1HMeEIDMTNxRODHYkRGIOhCBez83h3zSE+Xn8ELw83/jQijPsjW+Bpp0rHu07vYlb0LDYmbiSySSST2k2iX0g/3MRUUrYHRcNwS3SIny18fBbNyLhqvnh64t7Q2mF4Nm9OrW7dipmtGtn8G/URD/PxeT3MO2QwYH0wRUUn8delsZxMy2J8l2CeG96ORnXs8+3z51M/M2v3LDYnbaaedz3uansXP8b/yPQfptO0TlMmtpvI2DZjqeNlkjNLQvPyyEtOLj0M9+xZ8s6evRSGWxS32rUvm6nat7/SRNWwIe42s5V73brVzkzlTIyT3zj5azxFe7S0b+LPzDHt6dq8fqWvq6psO7mNWdGz2HZyG/V96vNA+we4++a7qe1Zm9yCXL4/9j1z989l5+md1PKoxejWo5nUbhKt6rayw525PgWZmZYwFPdnVCAMt8RIKptwuPmaMHJ7UlYnvxEYIzA1ltTMXN5afZDPNh2jjo8HTw+9mXu6N8O9kkUpVZVNSZuYvXs2O07vILBWIL8J/w13tr2TWh4lBwjEJMcwN3Yuy48sJ7cgt0qbzy6F4RbzZVQkDLdU8XCBMNyaihGYMmAEpmZSUKB8syOBN2w9WibZerTUq2QZfVVlQ+IGZkXPIvpMNI1qN+LB8AcZHzoeH4+ymdqSM5OZHzefefvncTrztEuZz64Iwy1NPKpBGK6hdIzAlAEjMDWP6ITzvLQohl3x5+nSrC4zx4QTHhxQqWuqKj8m/Mis3bOISY4hyDeIhzo8xNg2Y/Fyr5ho3UjzWdEw3CuEowJhuKWJR3UMw62pOERgRMQN8FPVtMoszlUwAlNzsHq07OfrbfE08PXm+eHtGFfJHi0FWsCa42uYHT2b2JRYgv2CmdphKqNbj7ZrCHJx81nvoN5MDpt8XfNZ8TDc/OLiUY4w3MviYcJwDXYUGFtByUeAfOBnIAD4p6q+aY+FOhMjMNWfvPwC5m49zj9sPVoeiGzB45Xs0VKgBaw6torZ0bOJOxdHszrNmBoxlRGtRuDp5rjcluTMZOYf/IaoHV+Rd/YsbQsCud23G53cm+N+7kIx8bh+GO5VPyYM11BG7JnJf4uqponIZGAZ8CyW0FR5gTFUb7bZerTEJqUR2boBM0ZXrkdLfkE+K4+u5IPoD/gl9Rda+Lfgb33/xvCWw/Fwq9yHcalhuEV2Hv3OnqXfpTDcU8BS0oBcHw+8Gjai1k1NTBiuwWUoy1+Fp6245Fisxl+5IlJzHTcGl+dUWhavLotl4a4TNAnw4f3JXRge3rjCH6x5BXksP7KcD6I/4GjaUVoHtOaN/m8wpPkQ3N3KV48s7+xZ0jdsIGP7dvJOFTFfpaRcvxpuy5ZXRVId8TjP/86uYknSKnILTtM7qDWTw8ZUyegzQ/WjLCayx7F2LbuBEUAz4AtV7ef45TkWYyKrXuTkFfDJT0d4e3UcufnKtP6tKtWjJbcgl6hfovhwz4fEX4gntF4oj0Q8wm3Nbyvzh7fm5ZEZHU36unVcXLeerH1Wx3D3unXxDA62WxhuSdFn99x8D2NDx+Lv5V+h+zcYroVDo8hExENV8yq0MhfCCEz1oWiPllvbNeLFSvRoyc3PZfEvi/lwz4ckpicSVj+Mhzs+zKCmg8okLLmnTnNxw3rS12/g4k8/UZCWBu7u1OrUCb9+ffHt1w+fsDCHhOLmFuTy/fHvmRtbc5M3DY7Hnk7+3wNzgAvAR0Bn4DlV/c4eC3UmRmCqPgnnMnglKpYVMVaPlj+PuoXB7SrWoyUnP4eFhxby0Z6PSLqYRPsG7Xm046P0D+lfqnlNc3PJ2LmTi+vXk75uPdkHDgDg0agRvv364tevP76RvXH3v7E7iX3J+5gbO5dlR5aVK/rMYLge9hSY3araUUSGAr8FXgTmqGoX+yzVeRiBqbpk5eYz+0erR4ubCNMHt+HBvi0r1KMlOz+b+Qfn8/HejzmdcZqIhhE82vFR+jTpc01hyU1KIn3dei5uWM/FnzZRcPEieHhQu0sX/Pr3w7dfP7zbtnUJh/ol89mBeZzOMOYzQ+Wxp8BEq2qEiLwNrFXVb0Vkp6p2ttdinYURmKpHYY+WmVExxKdkMiIiiBfuCKNJBXq0ZOZl8s3Bb5izdw5nMs/QpVEXHun4CL2Cel0lDAU5OWT+/DPp69aTvn4dOYd+AcAjKAi/fv3w69+P2r164e7nZ5f7dATGfGawF/YUmDlAMNAS6IjVqnitqna1x0KdiRGYqsXhM+n8Zck+fjx4htBGfvxldHsiK9CjJSM3g/8d+B9zYuaQkpVCj8Y9eKTjI3S7qdsVwpKTkGA559dv4OKWLWhGBuLpSe3u3fDt1x+/fn3xat3aJXYp5cWYzwyVwZ4C4wZ0Ag6r6nkRaQAEq2q0fZbqPIzAVA0Ke7R8tP4wPh7uPHF7W+7r3bzcPVou5l7kq/1f8VnMZ5zLPkfvoN483PFhut5kfVcqyMoiY9s20tev5+K69eQcPQqAZ0jIJbOXb8+euNWube9bdBrGfGaoCHaNIhOR0UB/2+GPqrqkkutzCYzAuDaqypLoJP5m69EyoUsIzw6/udw9Wi7kXGBu7Fw+j/2c1OxU+gb35eGIh+nYsCO5x47ZzF7rydi6Fc3ORry9qd2jB379+uHbry9eLVpUyV1KeTDmM0N5sOcO5jWgO/Cl7amJwHZVfb4MixgGvI1lVvtIVV8rdn4KVkWARNtT76rqR7Zzb2Dl3bgBq4Dfq6qKyFogCMi0zRmiqqcLwWDuAAAgAElEQVRF5CngISAPOAP8RlWPlbY+IzCuy/6Tafx5UQxbjqQQHuzPX0aH07V5vXJdIzU7lS9jv+SLfV9wIfcCA0MG8nDb+2l+6AIX128gff16cuPjAfBq0QLfQl9K9+41ur6WMZ8ZroddnfxAJ1UtsB27AztVNeI689yBg8DtQAKwDZioqvuKjJkCdFPV6cXmRmIJT+GuaQPwvKqutQnMH1R1e7E5g4AtqpohIo8CA1X1V6WtsaICcyErF3c3qXACn+HapGbm8q9VB/l8c8V7tJzPOs9n+z5j7v65XMxJZ4JXL+4+1wa/n+PI2LYNzc1FatXCt2dPfPv3w69fP7yaNnXgXVVNjPnMcC3sWYsMoC6QYntc1trmPYBDqnrYtqCvgTHAvlJnWSjgA3gBAnhiFV669gTVNUUONwO/LuM6y83/tifw16X7CG1Uh45NA4gIqUvHkLrc3LgOXh7mG15FKChQvvk5gddX7OdcRg6Tejbj/24vX4+W5MxkPtv3Gd/unkubXzJ55nQTwuK84ORGYCO5bVpTb/Jk/Pr3o1bXrrh5ezvuhqoBDWo1YFrENB4If+CS+ezN7W/y7q53jfnMUCbKIjCvAjtFZA3Wh31/4LrmMazIs/gixwlAzxLGTRCR/li7nSdVNV5VN9leL8n2mu+qamyROXNEJB+YD7yiV2/DHgSWl2GNFaJny/pMHxxKdMJ5Vu07xf+2JwDg5eFGWJA/nUJsotM0gFaBfpUqCV8T2B1/npcWx7A7/jxdm9fj09E9ytWj5UzGGRas+BdJ3y+l/aFc3k8A93zFrXYytSN74/dof/z69sEzONiBd1F98XTzZFiLYQxrMeyS+WxB3ALmHZhnzGeGUimrkz8Iyw8jwBbATVVPXGfOXcBQVX3Idnwv0ENVf1dkTAMgXVWzReQR4G5VHSwibbB8N4UmrlXAs6q6TkSCVTVRROpgCcwXqvpZkWv+GpgODFDVq9rqicg0YBpAs2bNuh47Vqqb5rqoKgnnMtmdcJ7ohFR2x59nT2IqGTlWYyY/bw/Cg/3p2NTa5USEBBBct1a1dxqXheT0bN5ceYB52+MJ9Lvco6Us701+Whon1q4gZukX1NlxiPoXrP/H0qYl9QcOxrdff2p37mRa6joIYz6r2Ti6FtlxVW12nTG9gRmqOtR2/DyAqr56jfHuQIqqBojI04CPqr5sO/cSkKWqbxSbM4UiPhwRuQ34N5a4nL7efTjKyZ9foPxyJp3d8ZboRCecJzbpAjn5BQA08PUiIiTgCtFp4FdzzDWFPVr+vvIAGTn5PNCnBY/fGkqdUnq0aEEB2fv3k75uPed+/IGc3XtwK1AuesPZ8GBCh99Ns9vH4HlTxcrEGCqGiT6rmThaYOJVtVSvqIh4YJm9bsWKEtsGTFLVmCJjglQ1yfZ4HNYupZeI/AqYCgzD2jWtAN7CMnvVVdWzthYCXwGrVXWWiHQGvgGGqWpcWe7jRkaRZeflsz/pAtEJ59ltE5240+mXKrQH161Fx6YBNsGpS4eQAPy8q18QwdYjKby0aC/7T16gT5sGzBjVntBr9GjJP3+e9I0brYivDRvIP3sWgCONhV2t3PDvP4DRY56mWb0WN/AODNfiWtFnfYP7lrutgcG1cfoOxjbuDixhcAf+q6p/FZGZWGHOi0XkVWA0VmhxCvCoqu637Wbex/L3KLBCVZ8SEV9gHZbT3x1YDTylqvkishrogOW3ATiuqqNLW5+zw5TTs/PYm5h6hejEp1jR1yLQuqEfHW2+nIiQuoQF1cHbo2r+oRbv0fLiyFsYVqxHixYUkLV376VEx8w9e6CgAPzrcKxdXZbddJLoVm7c2mkCD3Z4kCZ+TZx4R4ZrUdx8FuIXwsR2E435rBpRaYERkX9jfbhfdQq4X1Wr/P8UZwtMSSSnZxOdmEp0/GXhOZtuuZI83YV2jf2vMK+1aeRXrhDeG01OXgFzNh7hne/jyC1QHu7fiscGtqGWlyWUeSkpXNywwSptv2ED+efOgQg+HTqQ16MDUQ0T+Ux/ws3dgwltJ/Cb8N/Q2Lexk+/KUBaM+az6Yg+Bub+0iar6aQXX5jK4osAUR1U5kZpFdPzlXc6ehFQuZFvteGp7uRPeJOAK0Wla3zWCCNYdPMOMJTEcPnOR28KsHi3N6vqQGR19qbR9VkwMqOJevz6+ffvg168/Z8OD+TBhHsuPLMfLzYs7297JA+EP0Kh2I2ffkqGCFJrPlh9ZTk5BjjGfVXEcaiKrLlQFgSmJggLl8NmLRBdGriWcJ+ZEGjl5VhBBvdqedAipS8cQm0+naUC5y6tUhviUDF5Zuo+VMado0aA2f+nbmI5J+60mXBt/oiA1FdzcqNWx46V+KT7tb+FQ6i98EP0BK4+uxMfDh3tuvof72t9HYK3yF7Q0uCYpWSnMPzifrw98bcxnVRgjMGWgqgpMSeTmF3Dg5AUrXDreEp2Dpy5QYPv1BgX4EGHLz+nU1Aoi8C8laqsiZOXmM+vHX/jgh4O0SznGVO9TtD22h5z9+wFwbxiIX1+rHItv7964160LwIGUA8yOns2qY6uo7VGbSWGTuPeWe6nvU9+u6zO4DsZ8VrUxAlMGqpPAlERGTh4xJ9KuCJc+mpxx6XyrQN9LprWIkLq0b+JfoYZdqsoP6/ey6tOFtD6yh+7Jh/DOzgR3d2p37nypxpd3u3ZXmO5ikmOYvXs2a+LX4Ofpx+Swyfw67NfU9alrl/s3VA1KMp9NCptEv+B+xnzmohiBKQPVXWBK4nxGziWxKfTpnEqzggg83IS2N9W5Ily67U1+eJRQFl9zcsjYsYOElT9wctUaGp21qhkUNGhI/cEDrNL2vXvjXufqEOQ9Z/YwK3oW6xLWUcerDveG3cuksEkEeJc9e99Q/TDms6qDPYtdvlPC06lYocaLKrg+l6AmCkxJnErLYnf8+SuqEaRlWUEEPp5utLcFEXQK8qN90n5q/biKCz+sQTMyyBV39jdsRZ0B/ek3eRS+N1+7TfCu07uYtXsWG09sJMA7gPtuuY+J7SZSx6vkPBhDzaTQfPZV7FfsOL3DmM9cEHsKzAdAO+D/2Z6aAMQATbGakD1RybU6DSMwJaOqHEvOYHfCeXYfO8f57T8TsmMdkfG7CMjJIN2rNtuaduTHwHa0ur0/T43pTMM6165EsP3kdmZFz2JL0hbqedfj/vb3c0+7e/D19L2Bd2WoihjzmWtiT4H5AavnSp7t2AP4DqsM/x5VvcUO63UKRmCuTdbBg6RFLSUtKorcEycQHx+0d1+OdOrHxnqhnM3O5+EBrenSrOQeLarKtpPb+M/u/7D91HYa+DTggfAHuKvtXdT2rD4dIQ03BmM+cy3sKTAHsIpUptqOA7D6rrQTkZ2q2tkuK3YCRmCuJPfECVKXLiUtainZBw6Auzu+kZEEjByB36234e53/R2HqrIpaROzd89mx+kdNKzVkN+E/4YJbSdQy6PWDbgLQ3XGmM9cA3sKzIPAn4C1XC7X/zesOmAzVPXpSq/WSRiBgbxz57iw8jtSo5aQuf1nAGp16oT/yJH4DxuKR2DZclBUlQ2JG5gVPYvoM9HcVPsmHuzwIONDx+PtXnMKeRpuHMZ85jzsGkVmK9ffA0tgtl6vVH9VoaYKTEFmJhd++IG0qKWkb9gAubl4tWpFwKiR+I8cWa7ujqrKjwk/Mmv3LGKSYwjyDeKhDg8xts1YvNxNqXyD4zHmsxuPvQVmNJfbF/+oqksquT6XoCYJjOblcfGnn0iNiuLC6u/RjAw8broJ/xEjCBg5Au+wsHKVlynQAtYcX8Os6FnsT9lPsF8w0yKmMarVKDzd7ZvAaTCUhdyCXH44/gNzY+ca85mDsaeJ7DWsZmNf2p6aiBWiXJauli5NdRcYVSVz1y7LWb98OfkpKbj5++M/dCj+I0dSu1tXxL18poSLuRdZfmQ5c/fPJe5cHM3qNGNaxDTuaHUHnm5GWAyugTGfORZ7Ckw00ElVC2zH7sBOVY2wy0qdSHUVmOxDh0iNiiItaim5CQmItzd+gwYRMGokvv364VbOLo+qyu4zu1kQt4AVR1eQmZdJm7pteLDDgwxrMQwPt+rXt8ZQPTDmM8dgb4EZqKoptuP6wFojMK5F7smTpC1dSmrUUrJjY8HNDd/evfEfOZI6t9+Gu59fua+ZnJlM1OEoFsQt4HDqYWp51GJ4y+GMDx1PRGCES1RsNhjKgjGf2Rd7CsxE4DVgDZejyJ5X1a/tsVBnUtUFJv/8edK++460JVFkbN8OqvhERBAwciT+w4fh0bBh+a9ZkM+mpE0siFvAmvg15BXk0bFhR8aHjmdoi6EmOdJQ5THms8rjiCiy7lgCswVwqw6RZFVRYAqyskhfs4bUqKWkr1tnRYC1aIH/qJEEjByJV/PmFbpuYnoiCw8tZOGhhZy8eJJ63vUY1XoU40PH07puazvfhcHgfIz5rOK4RMtkV6eqCIzm5XFx8xbSlizhwqpVFGRk4NGwIf4jRuA/ciQ+7W+pkLkqJz+HH47/wIK4BWxO2gxAZJNIxoeOZ1DTQSYazFAjMOaz8uNogYlX1bInS7goriwwqkpWdDSphRFgZ8/iVqcOdYYOIWDkSGp3717uCLBC4s7FsSBuAVGHoziffZ4g3yDGtRnH2DZjCfILsvOdGAxVB2M+KxtmB1MGXFFgsg8fJi0qitSopeQeP454eeE3cCD+o0bi178/bt4Vy4ovDC/+Nu5bos9G4+HmweCmg5kQOoGeQT3NH4/BUARjPiudSguMiPwbKOmkAPer6nXfZREZBrwNuAMfqeprxc5PAd4EEm1PvauqH9nOvQGMANyAVcDvVVVFZC0QBGTa5gxR1dMi4g18BnQFkoFfqerR0tbnKgKTe+oUaUuXkRYVRda+fVYEWK+e+I8YSZ0ht5fYU6UsFIYXz4+bz8qjKy+FF49rM45RrUdRz6fkQpUGg8HiWuazie0m1mjfZFkFprQEhtI+ea/7qWzLl3kPq+pyArBNRBar6r5iQ+ep6vRicyOBPkBhKPQGYABWPTSAyapafA0PAudUtY2I3AO8Dvzqeut0FvlpaVz47jtSl0SRsXWrFQEWHs5Nzz9HneHD8WzUqMLXLgwvnh83nyOpR6jlUYs7Wt7BuNBxJrzYYCgHnm6eDG0xlKEthhKbHMvc/XP5Nu5b5h2YR6+gXkwOm2zMZ6VwTYFR1U+LPycijVX1ZBmv3QM4pKqHbXO/BsYAxQWmxJcHfAAvrB2TJ3DqOnPGADNsj78B3hURURdq2VmQnU36mrWkLY0ife2PaG4uXs2bE/jYY/iPHIF3y5YVvnZ+QT4/nfiJbw99y5rja8hTK7x4ZuRMhrYYakrkGwyVJKxBGC/3eZknuz55yXz2ux9+VzXNZ5nnwcsXHBzIU94U7GVAlzKODQbiixwnAD1LGDdBRPoDB4EnVTVeVTeJyBogCUtg3lXV2CJz5ohIPjAfeMUmIpdeT1XzRCQVaACcLfvt2R/NzydjyxZSl0RZEWDp6bg3DKTepIn4jxyFT3j7Su0oCsOLv437llMZp6jnXY9JYZNMeLHB4CDq+9RnasRUpoRPuWQ+e3P7m7y7613XN5+ln4bN78O2j+GON6HjPQ59ufIKTHk+CUsaW3w3sQT4SlWzReQR4FNgsIi0AcKAENu4VSLSX1XXYZnHEkWkDpbA3IvleynL6yEi04BpAM2aOSZOQVXJ2ruX1CVLrAiwM2dx8/OjzpAhBIwcQe2ePSscAQbXDi9+pvszJrzYYLhBVCnz2fnj8NO/YcdnkJcN7cdC4w4Of9lyRZGJyGOq+n4Zx/bG6hcz1Hb8PICqvnqN8e5AiqoGiMjTgI+qvmw79xKQpapvFJszBeimqtNFZKXt9TbZum6eBBqWZiKzt5M/+8iRS10gc44dQzw98Rs4AP+Ro/Ab0B83H59KXf/guYN8G/ctSw4vITU71YQXGwwuRknRZ/e0u4dxoeOcYz47cxA2vgXR86zjjvdAnycgMLRSl3VomHIZF+CBZfa6FStKbBswSVVjiowJUtUk2+NxwLOq2ktEfgVMBYZh7UxWAG8By4G6qnpWRDyxmp6tVtVZIvJboIOqPmJz8o9X1btLW6M9BCb39GnSli0jLWopWXv3ggi1e/YkYOQI6gwZgrt/5f5Tpeeks+LoChbELWDP2T14uHlwa7NbGd9mvAkvNhhcFKdHn53YBRv+CfsWg4cPdL0fek+HuvZJX3S6wNgWcQeWMLgD/1XVv4rITKxy/4tF5FVgNJAHpACPqup+227mfay6ZwqsUNWnRMQXWIfl9HcHVgNPqWq+iPgAnwOdbde6pzDA4FpUVGDyL1zgwnerSI1aQsaWrVBQgE/79lYXyDuG43nTTeW+ZlFUlV1ndrEgbsEV4cXjQ8czstVIE15sMFQhCs1nyw4vI6cgx3HmM1U49hOs/wf88j14B0CPqdDrUfAtW2fasuISAuPqVFRgzvz7Xc6+9x6ezZoRMNIq1+LdqvIlJZIzk1nyyxIWHFrAkdQj1Paofal6cYfADia82GCowjjMfKYKcassYYnfDLUDofdvofuD4BNgvxsoghGYMlBRgck9dYq8kyfxiah8TklhePGCuAWsjV9LnubRqWGnS9WLTXixwVC9sJv5rCAf9i2C9f+EU3vAPwT6/B46/xq8HPu5YQSmDDgzkz/hQsKl6sWF4cWjW49mXOg41w1xNBgMdqVC5rO8HIj+Gja8BSm/QINQ6PskdLgLPMrXTLCiGIEpAzdaYArDi+fHzWdz0mYEITI4kgmhExgYMtCEFxsMNZSUrBQWxC3gq/1fcTrjNMF+wUxsN/FK81nORSvM+Kd/Q1oiBHWEfv8H7UbCDQ72MQJTBm6UwBQPL27i24SxoWMZ29qEFxsMhsuUaD5rPpSJWQW03vEVZCRD8z7Q7ylofSs4yS9rBKYMOFJg0nPSWX7Uql685+wePN08GdxsMONDx9MrqBdu4uaQ1zUYDNWD2ISNzP3pVZZlHCVHhF7iy+TwB+jX6SGnpycYgSkD9haYwvDi+Qfn892x70x4scFgKD/nj8PGd2Dn55CXTcotI1kQfDNfnVh7bfPZDcYITBmwl8CY8GKDwVBpzhywHPd7/gdIkaz7NoALJG8WwQhMGaiMwJjwYoPBYBdO7LRCjWOX2LLup0DkdAgIueaUG5a8eQ2MwJSBigrM0sNL+dfP/+JUxinq+9RnVKtRjA8db/p3GwyGsqEKxzbasu5/sLLue06Dno+UK+u+TNFnDsAITBmoqMCsOb6GeQfnmfBig8FQPlQh7jtb1v0W8G1oZd13exB8Ki4IJZnPRrUaxaSwSQ4xnxmBKQOu0jLZYDBUcwryYd9CWP8vK+s+oOnlrHvPWnZ9qRthPjMCUwaMwBgMBoeSlw27v7ZK5qcchsC2l7PuHWz5KDSffb3/a05lnLKr+cwITBkwAmMwGBxCzkX4+VMr6/7CCQjqVCTr/sbmwF3LfPbrW35Ny4CKtWkvq8CUt6OlwWAwGK5F5jnY+pHVljgzBZr3hTHvQuvBTsu6L6nz5sJDCwkPDK+wwJQVs4MxOxiDwVBZ0k/DpvesXvc5F6DtMOj7FDTr6eyVlUhKVgq+nr54u3tXaL7ZwRgMBoOjOXcMfnoHdn4B+TnQfpzlY7kB/e4rQ32f+jfkdYzAGAwGQ3k5cwA2/Aui/wfiBp0mWln3DUyrjaIYgTEYDIaykrjD6nUfG2WFF/d82Op1HxDs7JW5JEZgDAaDoTRU4egGKzny8BqrDXH/p21Z9w2cvTqXxgiMwWAwlIQqHFxpCUvCVvBtBLf9Bbr9plJZ9zUJhwZki8gwETkgIodE5LkSzk8RkTMissv281CRc2+ISIyIxIrIO1KsJLGILBaRvUWOO4nIZtt1totID0fem8FgqKYU5MOeb2BWX/jqV3DhJNzxd3giGvo+YcSlHDhsByMi7sB7wO1AArBNRBar6r5iQ+ep6vRicyOBPkCE7akNwABgre38eCC92HXeAP6iqstF5A7b8UC73ZDBYKje5GXD7q9g49u2rPubYdxsCJ/g8Kz76oojTWQ9gEOqehhARL4GxgDFBaYkFPABvAABPIFTtuv4AU8B04D/FZtT+NUiADhR+VswGAzVnpyL8PMntqz7JCvr/ldfwM0jbnjWfXXDkQITDMQXOU4ASso6miAi/YGDwJOqGq+qm0RkDZCEJTDvqmqsbfzLwD+AjGLXeQJYKSJ/xzL9RdrvVgwA5OdC5nnIOg9u7lDftCcwVGEyz8HWD2Hzf6ys+xb9YOz70GqQ07LuqxuOFJiSfkPFywYsAb5S1WwReQT4FBgsIm2AMKCw484qmwilAW1U9UkRaVHsWo9iCdR8Ebkb+Bi47apFiUzD2v3QrFmzCt1YlSYv+7JIXO/frNQrn8u9eOW1bhkLQ16GujXwfTRUXS6cgs2FWffpLp91X5VxWKkYEekNzFDVobbj5wFU9dVrjHcHUlQ1QESeBnxU9WXbuZeALOAC8CKQgyWOjYCfVHWgiKQCdVVVbQEBqapaqjeuypaKyc0so0ikXv1cXmbp1/byA5+6UKtuCf8GXH6c/Itlq0Yh8nHL+enle0Nu32CoEOeO2nrdfwEFudB+vC3rPtzZK6tyuEKpmG1AqIi0BBKBe4BJRQeISJCqJtkORwOFZrDjwFQReRVrJzQAeEtVlwD/sc1tAUSp6kDbnBNcDgQYDMQ54qbsgirkZpR/B1H4b3526df39rcJgU0QAtuUIBL1ShaQ8jgzu9wLq/4M696AXV/C7TMth6gxLxhcidP7raz7Pf/PlnU/yerFYrLuHY7DBEZV80RkOrAScAf+q6oxIjIT2K6qi4HHRWQ0kAekAFNs07/BEok9WGa1FTZxKY2pwNsi4oG125lm73u6AlVre13eHUThvwW5pVxcrFDIogLQqJ31r0/ANXYXNtHw9gf3G5TeFBACd34M3R+CFc/C/Actm/bw16FJpxuzBoPhWiT+bPW63x8FnrWtxMjevzVZ9zcQU025Iiayn96F1X+GgrxrjxG3K01KJZmZrvWvt7/lRK9KFORbpofvZ0JGsrW7GfwS+DV09soMNQlVOLrelnW/1vp76/kI9HjYZN3bEdNwrAxUWGCOboBD35cuEl51amaIY1Yq/PgGbJllfWsc8Cz0mAYeXs5emaE6owoHV9iy7rdZWfeR06HrAyYx0gEYgSkDVdbJXxU4GwcrnodDq6BBKAx7FUJvd/aqDNWN/Dxbr/t/wukYK6Kxz++h06/B08fZq6u2uIKT31CTCQyFX38DB7+Dlc/Dl3dC6FAY+jcr6MBgqAyFWfcb3oJzR6BhOxj3AYSPN1n3LoQRGINjaTsEWg2ErbMt09n7vawS5wOesezjBkN5yE63su43vWtl3TfpDEO+hJvvqJkmaRfHCIzB8Xh4QeTvIOJXVhDApvcgeh7c+mfoNNl8MBiuT0aKFaG45T9WBn6LfjD2P9aXFxMW77IYH4zxwdx4TuyE5c9C/Bar7tPwN0wWtaFkLpy0vpBs/68t63449HsKmppi6c7E+GAMrkuTzvCblVZJ9FUvwX+HQIe7rF4bJkfBAFdn3YdPsLLub2rv7JUZyoERGINzEIGIu6DdHVaW9cZ3YP9SqyZU5HSrHa2h5nE61pZ1/42VC1aYdW8Kq1ZJjInMmMhcg3PH4Ls/QexiK9R0yCsQNtrY12sKCT9bve4Ls+67/cbKuvdv4uyVGUrAmMgMVYt6zeFXn8ORdbD8OfjffZYjd9hrphhhdUXV+n2v/wcc+dFKUh7wrJV5X7u+s1dnsANGYAyuRcv+8PA62PEJ/PAKzO5nfZsd9IL50KkuFBRczrpP3A5+N8HtL0O3B8C7jrNXZ7AjRmAMroe7h1VAs/14WPsabPvIsskPesESmxtVzNNgX/LzIOZbyxR2ep9lCh3xTytU3WTdV0uMD8b4YFyfU/tgxXOWGaVhGAx/zcp/MFQN8rJh11zY+JYVHdawnRXMET7BfFmoohgfjKH6cNMtcN8iK8ps5R/hszHQbqQVCFC/pbNXZ7gW2enw8xyr+nj6SWjSBYb81WTd1yCMwBiqBiIQNhLa3Ga1u133D3ivB/SeDv3+D7z9nL1CQyEZKbD1A6uiduY5y682fja0HGCiAmsYxkRmTGRVk7QkWD0Dor8Gv8Zw+1+gw93mm7EzuXDSqhG27b+Qe9HaqfR9Cpp2d/bKDHbGlOsvA0ZgqgHx22D5M3BiB4R0h2GvQ0hXZ6+qZpFyBH4qzLrPg/A7bVn3tzh7ZQYHYXwwhppB0+7w0PdW6fbVM+CjwVZU0q0vQZ3Gzl5d9ebUPivrfu98W9b9ZOjzuMm6N1zCCIyh6uPmBp0nQ9goK7di8/uwbxH0fxp6PQoe3s5eYfUiYbvV4OvAUvD0td7j3tPBP8jZKzO4GMZEZkxk1Y/kX6yyMweWWd+mh/4N2g4zDubKoGqFia//h5V971PXyrjv+bBJgK2BlNVE5lCPqIgME5EDInJIRJ4r4fwUETkjIrtsPw8VOfeGiMSISKyIvCNy5aeDiCwWkb3Fnvud7fViROQNx92ZwaVp0BomfgW/ng9unvDVPfDFeDhzwNkrq3oUFFjh4R/daoWHnzlohYc/uRcGPW/ExVAqDjORiYg78B5wO5AAbBORxaq6r9jQeao6vdjcSKAPEGF7agMwAFhrOz8eSC82ZxAwBohQ1WwRaWTfOzJUOdrcBo8OsCoBrHkV3u8NPabBwOegVl1nr861yc+DmAWWKexMLNRtDiP/BR0nmax7Q5lx5A6mB3BIVQ+rag7wNZYAlAUFfAAvwBvwBE4BiIgf8BTwSrE5jwKvqWo2gKqervQdGKo+7p6Wj+DxHdDlPis3499drAZWBfnOXp3rkZtlvTf/7gILplrPjf8QfrfDKtNjxMVQDhwpMMFAfJHjBNtzxZkgItEi8lUUIfMAAA0BSURBVI2INAVQ1U3AGiDJ9rNSVWNt418G/gFkFLtOW6CfiGwRkR9FxATfGy7jGwij3rIKaTZsB1FPwuwBcHSjs1fmGmRfsHryvB1hvTe+gXDPXHj0J4i425R0MVQIRwpMSR7V4hEFS4AWqhoBrAY+BRCRNkAYEIIlSoNFpL+IdALaqOq3JVzbA6gH9AKeBv5X3G9ju/Y0EdkuItvPnDlTwVszVFmCImDKUrhzjpVl/skd8P+mwPnjzl6Zc8hIscyH/wqHVS9a4nvfYiv0u90Ik7hqqBSO/FqSADQtchwCnCg6QFWTixx+CLxuezwO2Kyq6QAishxLOC4AXUXkKNbaG4nIWlUdaHu9BWqFxW0VkQIgELhCRVT1A+ADsKLIKn+bhiqHCISPtyLLfvq3lctxYDn0ecLqnuhV29krdDxpSVbW/fY5tqz7EVav+5DrBgYZDGXGkV9PtgGhItJSRLyAe4DFRQeISNHA+dFAoRnsODBARDxExBPLwR+rqv9R1Saq2gLoCxy0iQvAQmCw7bptsfw3Zx1yZ4bqgVdtGPgsTN9mlTX58TV4t7uVOFhdw/dTjsCSJyxT2Ob/WPXdHt0EE+cacTHYHYftYFQ1T0SmAysBd+C/qhrz/9u792CryjKO499fopJiCuIFFQWKUDEMNMpr0OAA5iXTRi0N1KzMNMemxssMOVpmmmmmTpTiaBk20k1NUUfwBiKgqXgUUUGTaIgkUvIW+PTH+55peTqXfdhnrb3N32fmzFlnrbXPfs573n2es9a73+eVdD6wMCJuAU6XdBiwDlgNTM4Pn0FKFotIt9VmRsStXTzlNGBafuvyW8CkeC9P8rHabTUQPncdjD45lZ2ZcSLMvyYtCzBgz0ZH1zNWthRm3feCkcfBvqe7GrWVyhMtPdHSit5eD3/6BdxzfhqfGPXFVHZm8/6NjmzDvLQgLfD1zO1p1v3HToRPnOpZ91YX1yIz2xDv2wj2mgy7fwbuuxjmT4WW36dbaaO/nN723OwiYOm9KbEsux/e3xfGnJ3i98RIq5CvYHwFY51Z9QzMPBuevwf6fxjGfx+Gjmt0VO17++10pfLApam6dJ/tYd/TUsL0ejnWg3wFY9YTthmWSs4suRPuPBtuPDK9+2z8hakkTTNYvy6NrTz4I1i1GPoOgkMuh49+3oU+raGcYMy6IsGwCfDBsakSwH2XwFUfTxUCDvwW9P5AY+L69xvw2I0w58ew5kXYdnf47DUw/AhPjLSm4F5oVqtem6Z5MiOOSW8CmHsFPH4TjPtOqtFV1aTEN19N5VweugrWroQd94aJP4Ch4z0x0pqKx2A8BmMb6i+PwB1nwfL5sMNImHgxDBxd3vO9tjpdQT08Fd5YA0PGwAHfhEEHeCkCq5THYMzKtuNecNJdsOhmuHsKXHsQjDgaxp0HH9ih557nlRXpaqV11v2uh6S17r00tDU5JxizekipGOSwg9NExrk/gadvTWVX9jmtvurDq5em8ZXHfpXm53zkc7D/GbDtbj0Xv1mJfIvMt8isJ61elopGPn1rWkNl/PfSFUd3bmGtbEnrsLT8Ni2YNvK4tNZ930GlhW3WHb5FZtYI/QbD0b9MEx1nng2/Pg4GHwgTLoLthnf+2JcWpDksS+6ATfqkde73ORW22L6S0M16mhOMWRmGjIGvPACPXAezvgs/3R/2PgnGnvPO2fSts+4fuBReeCDPuj8n1UXzrHt7l3OCMSvLRr1SotjjSJh9ISy8Fp6cAWPPTbPrl8xMt8JWPApbDEiTN0dN8qx7+7/hMRiPwVhVVrbAzLNSfbBN+sBba6Hv4DRwv+exnnVv7xoegzFrNtsNT6tFLr4NFs2A3Q5NRTU9697+T7lnm1VJSollt0MbHYlZ6VxXwszMSuEEY2ZmpXCCMTOzUjjBmJlZKZxgzMysFE4wZmZWCicYMzMrhROMmZmV4j1dKkbSKuDFRsfRgf7A3xsdRCccX32aPT5o/hgdX33qiW+XiNimq5Pe0wmmmUlaWEutn0ZxfPVp9vig+WN0fPWpIj7fIjMzs1I4wZiZWSmcYJrXzxodQBccX32aPT5o/hgdX31Kj89jMGZmVgpfwZiZWSmcYBpA0gRJz0h6TtJZ7Rw/U9JTkp6QdI+kXQrH1kt6LH/c0qD4JktaVYjjS4VjkyQ9mz8mNSi+ywqxLZG0pnCsivabJulvkp7s4LgkXZHjf0LSqMKxUtuvhti+kGN6QtJcSXsWjr0gaVFuu9KWgq0hxjGS/ln4PU4pHOu0b1QU37cKsT2Z+1y/fKzUNpQ0UNJsSU9LapH0jXbOqa7/RYQ/KvwANgKeB4YAmwCPA7u3OWcssFnePgX4deHY2iaIbzJwZTuP7QcszZ/75u2+VcfX5vzTgGlVtV9+jgOBUcCTHRw/GLgDEPAJ4OEK26+r2PZtfU5gYmts+esXgP5N0H5jgNvq7Rtlxdfm3EOBWVW1ITAAGJW3twCWtPP6raz/+QqmeqOB5yJiaUS8BdwEHF48ISJmR8Rr+ct5wE7NFF8nxgN3R8TqiPgHcDcwocHxHQtM7+EYOhUR9wOrOznlcOCGSOYBW0kaQAXt11VsETE3PzdU3/daY+iq/TpST9+tWTfjq7T/RcRfI+LRvP0q8DSwY5vTKut/TjDV2xF4qfD1cv63AxSdRPpvo1VvSQslzZP0mQbGd2S+vJ4haWA3H1tFfORbi4OBWYXdZbdfLTr6Gapov+5o2/cCuEvSI5K+3KCYWu0j6XFJd0ganvc1VftJ2oz0B/o3hd2VtaGkQcBI4OE2hyrrf73qebBtELWzr9238kk6Dtgb+GRh984RsULSEGCWpEUR8XzF8d0KTI+INyV9Fbge+FSNj60ivlbHADMiYn1hX9ntV4uOfoYq2q8mksaSEsz+hd375bbbFrhb0uL833zVHiWVKlkr6WDg98BQmqj9skOBORFRvNqppA0l9SEltjMi4pW2h9t5SCn9z1cw1VsODCx8vROwou1JksYB5wKHRcSbrfsjYkX+vBS4l/QfSqXxRcTLhZh+DuxV62OriK/gGNrcnqig/WrR0c9QRft1SdII4Brg8Ih4uXV/oe3+BvyOdEuqchHxSkSszdu3AxtL6k+TtF9BZ/2vtDaUtDEpudwYEb9t55Tq+l9Zg03+6HAQrhdp8Gww/x2IHN7mnJGkwcqhbfb3BTbN2/2BZ+nhQcwa4xtQ2D4CmJe3+wHLcpx983a/quPL5w0jDaiqyvYrPNcgOh6k/jTvHGSdX1X71RDbzsBzwL5t9m8ObFHYngtMKKPtaohx+9bfK+kP9J9zW9bUN8qOLx/fkjROs3mVbZjb4Qbg8k7Oqaz/+RZZxSJinaSvA3eS3vUyLSJaJJ0PLIyIW4BLgD7AzZIA/hwRhwG7AVMlvU26+rwoIp5qQHynSzoMWEd6EU3Oj10t6QJgQf5258c7bw9UFR+kwdWbIr9ystLbD0DSdNI7nfpLWg58B9g4x/9T4HbSO3meA14DTsjHSm+/GmKbAmwNXJ373rpIBRG3A36X9/UCfhURM3sytm7EeBRwiqR1wOvAMfn33G7faEB8kP7xuisi/lV4aBVtuB9wPLBI0mN53zmkfxwq73+eyW9mZqXwGIyZmZXCCcbMzErhBGNmZqVwgjEzs1I4wZiZWSmcYMzMrBROMGbvArlE/W31nmNWJScYsx6Q19jw68mswC8Isw0kaVBe2OlqUgHG4yU9JOlRSTfngoNIOljSYkkP5oWeOrzKkDRaaaGvP+XPw9o55zxJv5A0Ky8MdXLhcJ9c4XqxpBuVp41LmiJpQV4A62et+83K5ARjVp9hpNpPB5GqD4+LiFHAQuBMSb2BqcDEiNgf2KaL77cYODAiRpLKtlzYwXkjSDWl9gGmSNoh7x8JnAHsTlp4a7+8/8qI+FhE7AG8Hzik2z+pWTe5FplZfV6MiHmSDiH9UZ+TLw42AR4CdgWWRsSyfP50oLN1QLYErpc0lFQqfeMOzvtDRLwOvC5pNqno4xpS4cLlALkW1SDgQWCspG8Dm5GKGraQll0wK40TjFl9WosZirQa4LHFg5K6uxzABcDsiDgiLxh1bwfntS0i2Pr1m4V964Fe+SrqamDviHhJ0nlA727GZdZtvkVm1jPmAftJ+hCk1QwlfZh0y2tIThYAR3fxfbYE/pK3J3dy3uGSekvamlTZd0En57Ymk7/ncaGjuojBrEc4wZj1gIhYRUoI0yU9QUo4u+bbWF8DZkp6EFgJ/LOTb3Ux8H1Jc0gl5zsyH/hjfp4LIi9k1UFsa0gLwy0irf7YWTIy6zEu129WMkl9Ii3vK+Aq4NmIuKyO73cesDYifthTMZqVwVcwZuU7OQ+4t5BugU1tcDxmlfAVjFkDSDoB+Eab3XMi4tRGxGNWBicYMzMrhW+RmZlZKZxgzMysFE4wZmZWCicYMzMrhROMmZmV4j9QskWViygJMAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb1efe10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  优选结果： {'reg_alpha': 1, 'reg_lambda': 1},\n",
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
